import requests
import time
import threading
from lxml import etree
import pandas as pd
import re
from concurrent.futures import ThreadPoolExecutor, as_completed
from requests.exceptions import RequestException

class ProxyPool:
    def __init__(self, proxy_pool_url="http://127.0.0.1:5010"):
        self.proxy_pool_url = proxy_pool_url
        
    def get_proxy(self):
        """获取一个代理"""
        try:
            response = requests.get(f"{self.proxy_pool_url}/get/")
            if response.status_code == 200:
                return response.json().get("proxy")
        except:
            return None
            
    def delete_proxy(self, proxy):
        """删除一个代理"""
        try:
            requests.get(f"{self.proxy_pool_url}/delete/?proxy={proxy}")
        except:
            pass
            
    def get_all_proxies(self):
        """获取所有代理"""
        try:
            response = requests.get(f"{self.proxy_pool_url}/all/")
            if response.status_code == 200:
                return response.json()
        except:
            return []

cookies = {
    'sajssdk_2015_cross_new_user': '1',
    'sensorsdata2015jssdkchannel': '%7B%22prop%22%3A%7B%22_sa_channel_landing_url%22%3A%22%22%7D%7D',
    'wz_uuid': 'X/fe36031e2352d91d31adfe63416a305b',
    'Hm_lvt_9ea3e7293b7c088e0d2c88874b63e7dd': '1747617767',
    'HMACCOUNT': 'A735DDA42AC02E6E',
    'SSO_SESSION_ID': 'ZWVjOGY0MmYtZDExMi00YWZmLTg4ZWQtZDdlNDVkYjMwMWY0',
    'ticket': 'eyJhbGciOiJIUzUxMiJ9.ZXlKNmFYQWlPaUpFUlVZaUxDSmhiR2NpT2lKa2FYSWlMQ0psYm1NaU9pSkJNVEk0UTBKRExVaFRNalUySW4wLi5odVhicExqVDBmTG1STXQ2b1ZPYnZnLkp2R3BYa2FlcHVaN1ZjUHkySjBheVRNVlZyWURSdmhmazREdWhrc2s2RTQwQWZvVDNhZXA0REkyRVF1ek1zekJOZEM2STFqZzJOSTJjRV9GX3VRZlZDNVZDWEFQSmtrS2tMSTZpSW5CM0FjYU82TFNjV2hrUTVva1htN29UTU9SbjZCVWpmcDdyRnl3RG5XRHo2NDRiSm1vT28xMGx2VnFGczM2OUlmeUltVmlfZjFmMU5GaHVYTk1mSzJrd2lVX2xGaUR5YThmd3dLZFpCaG1Wa3NMN3ktWnVMNDIzOWk3aUJYSHpTTkw5NjJSZVI1aTBPNjJyeEk1SUZmeWtrS0IuRGd4aE1TSE5YcHNRdlRiX3VZVXpxUQ.K0Ytme5n24LKre-EXu0nz7CClT1iT2HX_bt_vmFnppc3xwzHCGia43d3WyE3JZVksuo_N5EjcDxj3s3HgI-W8g',
    '_g_t_': '96f67a30d3211b400f55ae4ed110f261',
    'token': 'eyJhbGciOiJIUzUxMiJ9.ZXlKNmFYQWlPaUpFUlVZaUxDSmhiR2NpT2lKa2FYSWlMQ0psYm1NaU9pSkJNVEk0UTBKRExVaFRNalUySW4wLi5odVhicExqVDBmTG1STXQ2b1ZPYnZnLkp2R3BYa2FlcHVaN1ZjUHkySjBheVRNVlZyWURSdmhmazREdWhrc2s2RTQwQWZvVDNhZXA0REkyRVF1ek1zekJOZEM2STFqZzJOSTJjRV9GX3VRZlZDNVZDWEFQSmtrS2tMSTZpSW5CM0FjYU82TFNjV2hrUTVva1htN29UTU9SbjZCVWpmcDdyRnl3RG5XRHo2NDRiSm1vT28xMGx2VnFGczM2OUlmeUltVmlfZjFmMU5GaHVYTk1mSzJrd2lVX2xGaUR5YThmd3dLZFpCaG1Wa3NMN3ktWnVMNDIzOWk3aUJYSHpTTkw5NjJSZVI1aTBPNjJyeEk1SUZmeWtrS0IuRGd4aE1TSE5YcHNRdlRiX3VZVXpxUQ.K0Ytme5n24LKre-EXu0nz7CClT1iT2HX_bt_vmFnppc3xwzHCGia43d3WyE3JZVksuo_N5EjcDxj3s3HgI-W8g',
    'accessToken': 'eyJhbGciOiJIUzUxMiJ9.ZXlKNmFYQWlPaUpFUlVZaUxDSmhiR2NpT2lKa2FYSWlMQ0psYm1NaU9pSkJNVEk0UTBKRExVaFRNalUySW4wLi5odVhicExqVDBmTG1STXQ2b1ZPYnZnLkp2R3BYa2FlcHVaN1ZjUHkySjBheVRNVlZyWURSdmhmazREdWhrc2s2RTQwQWZvVDNhZXA0REkyRVF1ek1zekJOZEM2STFqZzJOSTJjRV9GX3VRZlZDNVZDWEFQSmtrS2tMSTZpSW5CM0FjYU82TFNjV2hrUTVva1htN29UTU9SbjZCVWpmcDdyRnl3RG5XRHo2NDRiSm1vT28xMGx2VnFGczM2OUlmeUltVmlfZjFmMU5GaHVYTk1mSzJrd2lVX2xGaUR5YThmd3dLZFpCaG1Wa3NMN3ktWnVMNDIzOWk3aUJYSHpTTkw5NjJSZVI1aTBPNjJyeEk1SUZmeWtrS0IuRGd4aE1TSE5YcHNRdlRiX3VZVXpxUQ.K0Ytme5n24LKre-EXu0nz7CClT1iT2HX_bt_vmFnppc3xwzHCGia43d3WyE3JZVksuo_N5EjcDxj3s3HgI-W8g',
    'creditNo': '%22%22',
    'param_sign': 'roYuB3',
    'Hm_lpvt_9ea3e7293b7c088e0d2c88874b63e7dd': '1747619300',
    'smidV2': '202505191016307c10a18d5e8a15a50dda3fc1039d489f00ea7a21ddcd9e570',
    '_bl_uid': 'smmtqayquXdgR4fq7006g2h9IX2I',
    'Hm_lvt_8b5ea374777742fd73d0561bd1dfca6d': '1747620994',
    'acw_tc': 'f5d36357174762107277210433481bd19eb244908197885b08a7a18a2916d7',
    'sensorsdata2015jssdkcross': '%7B%22distinct_id%22%3A%22f7b1dcc0ddbf455d833fa99414f2780e%22%2C%22first_id%22%3A%22196e622d7c2e7c-0287f8b21c83c1e-26011f51-1327104-196e622d7c31b05%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22identities%22%3A%22eyIkaWRlbnRpdHlfY29va2llX2lkIjoiMTk2ZTYyMmQ3YzJlN2MtMDI4N2Y4YjIxYzgzYzFlLTI2MDExZjUxLTEzMjcxMDQtMTk2ZTYyMmQ3YzMxYjA1IiwiJGlkZW50aXR5X2xvZ2luX2lkIjoiZjdiMWRjYzBkZGJmNDU1ZDgzM2ZhOTk0MTRmMjc4MGUifQ%3D%3D%22%2C%22history_login_id%22%3A%7B%22name%22%3A%22%24identity_login_id%22%2C%22value%22%3A%22f7b1dcc0ddbf455d833fa99414f2780e%22%7D%2C%22%24device_id%22%3A%22196e622d7c2e7c-0287f8b21c83c1e-26011f51-1327104-196e622d7c31b05%22%7D',
    'x-web-ip': '171.218.206.136, 47.121.110.187, 47.106.31.167, 100.121.99.209',
    '.thumbcache_de0d870e3139ba2368b2e7ea8f11063c': 'hZVjd47Cv1sJHhGMmTvVn0f9DpivVn6IF/IytbVBOsjFSd0QEKKRar19TbnFCtp96Hi7/tlo0p6ovKPk4j5lOw%3D%3D',
    'Hm_lpvt_8b5ea374777742fd73d0561bd1dfca6d': '1747621872',
}

headers = {
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
    'accept-language': 'zh-CN,zh;q=0.9',
    'cache-control': 'max-age=0',
    'priority': 'u=0, i',
    'referer': 'https://qiye.qizhidao.com/lib/a01b00c00d00e00f00g00h00i00k00/sc2d10dc5af79548fbff85fbd1b22bf982/',
    'sec-ch-ua': '"Chromium";v="136", "Google Chrome";v="136", "Not.A/Brand";v="99"',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-platform': '"Windows"',
    'sec-fetch-dest': 'document',
    'sec-fetch-mode': 'navigate',
    'sec-fetch-site': 'same-origin',
    'sec-fetch-user': '?1',
    'upgrade-insecure-requests': '1',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36',
    # 'cookie': 'sajssdk_2015_cross_new_user=1; sensorsdata2015jssdkchannel=%7B%22prop%22%3A%7B%22_sa_channel_landing_url%22%3A%22%22%7D%7D; wz_uuid=X/fe36031e2352d91d31adfe63416a305b; Hm_lvt_9ea3e7293b7c088e0d2c88874b63e7dd=1747617767; HMACCOUNT=A735DDA42AC02E6E; SSO_SESSION_ID=ZWVjOGY0MmYtZDExMi00YWZmLTg4ZWQtZDdlNDVkYjMwMWY0; ticket=eyJhbGciOiJIUzUxMiJ9.ZXlKNmFYQWlPaUpFUlVZaUxDSmhiR2NpT2lKa2FYSWlMQ0psYm1NaU9pSkJNVEk0UTBKRExVaFRNalUySW4wLi5odVhicExqVDBmTG1STXQ2b1ZPYnZnLkp2R3BYa2FlcHVaN1ZjUHkySjBheVRNVlZyWURSdmhmazREdWhrc2s2RTQwQWZvVDNhZXA0REkyRVF1ek1zekJOZEM2STFqZzJOSTJjRV9GX3VRZlZDNVZDWEFQSmtrS2tMSTZpSW5CM0FjYU82TFNjV2hrUTVva1htN29UTU9SbjZCVWpmcDdyRnl3RG5XRHo2NDRiSm1vT28xMGx2VnFGczM2OUlmeUltVmlfZjFmMU5GaHVYTk1mSzJrd2lVX2xGaUR5YThmd3dLZFpCaG1Wa3NMN3ktWnVMNDIzOWk3aUJYSHpTTkw5NjJSZVI1aTBPNjJyeEk1SUZmeWtrS0IuRGd4aE1TSE5YcHNRdlRiX3VZVXpxUQ.K0Ytme5n24LKre-EXu0nz7CClT1iT2HX_bt_vmFnppc3xwzHCGia43d3WyE3JZVksuo_N5EjcDxj3s3HgI-W8g; _g_t_=96f67a30d3211b400f55ae4ed110f261; token=eyJhbGciOiJIUzUxMiJ9.ZXlKNmFYQWlPaUpFUlVZaUxDSmhiR2NpT2lKa2FYSWlMQ0psYm1NaU9pSkJNVEk0UTBKRExVaFRNalUySW4wLi5odVhicExqVDBmTG1STXQ2b1ZPYnZnLkp2R3BYa2FlcHVaN1ZjUHkySjBheVRNVlZyWURSdmhmazREdWhrc2s2RTQwQWZvVDNhZXA0REkyRVF1ek1zekJOZEM2STFqZzJOSTJjRV9GX3VRZlZDNVZDWEFQSmtrS2tMSTZpSW5CM0FjYU82TFNjV2hrUTVva1htN29UTU9SbjZCVWpmcDdyRnl3RG5XRHo2NDRiSm1vT28xMGx2VnFGczM2OUlmeUltVmlfZjFmMU5GaHVYTk1mSzJrd2lVX2xGaUR5YThmd3dLZFpCaG1Wa3NMN3ktWnVMNDIzOWk3aUJYSHpTTkw5NjJSZVI1aTBPNjJyeEk1SUZmeWtrS0IuRGd4aE1TSE5YcHNRdlRiX3VZVXpxUQ.K0Ytme5n24LKre-EXu0nz7CClT1iT2HX_bt_vmFnppc3xwzHCGia43d3WyE3JZVksuo_N5EjcDxj3s3HgI-W8g; accessToken=eyJhbGciOiJIUzUxMiJ9.ZXlKNmFYQWlPaUpFUlVZaUxDSmhiR2NpT2lKa2FYSWlMQ0psYm1NaU9pSkJNVEk0UTBKRExVaFRNalUySW4wLi5odVhicExqVDBmTG1STXQ2b1ZPYnZnLkp2R3BYa2FlcHVaN1ZjUHkySjBheVRNVlZyWURSdmhmazREdWhrc2s2RTQwQWZvVDNhZXA0REkyRVF1ek1zekJOZEM2STFqZzJOSTJjRV9GX3VRZlZDNVZDWEFQSmtrS2tMSTZpSW5CM0FjYU82TFNjV2hrUTVva1htN29UTU9SbjZCVWpmcDdyRnl3RG5XRHo2NDRiSm1vT28xMGx2VnFGczM2OUlmeUltVmlfZjFmMU5GaHVYTk1mSzJrd2lVX2xGaUR5YThmd3dLZFpCaG1Wa3NMN3ktWnVMNDIzOWk3aUJYSHpTTkw5NjJSZVI1aTBPNjJyeEk1SUZmeWtrS0IuRGd4aE1TSE5YcHNRdlRiX3VZVXpxUQ.K0Ytme5n24LKre-EXu0nz7CClT1iT2HX_bt_vmFnppc3xwzHCGia43d3WyE3JZVksuo_N5EjcDxj3s3HgI-W8g; creditNo=%22%22; param_sign=roYuB3; Hm_lpvt_9ea3e7293b7c088e0d2c88874b63e7dd=1747619300; smidV2=202505191016307c10a18d5e8a15a50dda3fc1039d489f00ea7a21ddcd9e570; _bl_uid=smmtqayquXdgR4fq7006g2h9IX2I; Hm_lvt_8b5ea374777742fd73d0561bd1dfca6d=1747620994; acw_tc=f5d36357174762107277210433481bd19eb244908197885b08a7a18a2916d7; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22f7b1dcc0ddbf455d833fa99414f2780e%22%2C%22first_id%22%3A%22196e622d7c2e7c-0287f8b21c83c1e-26011f51-1327104-196e622d7c31b05%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22identities%22%3A%22eyIkaWRlbnRpdHlfY29va2llX2lkIjoiMTk2ZTYyMmQ3YzJlN2MtMDI4N2Y4YjIxYzgzYzFlLTI2MDExZjUxLTEzMjcxMDQtMTk2ZTYyMmQ3YzMxYjA1IiwiJGlkZW50aXR5X2xvZ2luX2lkIjoiZjdiMWRjYzBkZGJmNDU1ZDgzM2ZhOTk0MTRmMjc4MGUifQ%3D%3D%22%2C%22history_login_id%22%3A%7B%22name%22%3A%22%24identity_login_id%22%2C%22value%22%3A%22f7b1dcc0ddbf455d833fa99414f2780e%22%7D%2C%22%24device_id%22%3A%22196e622d7c2e7c-0287f8b21c83c1e-26011f51-1327104-196e622d7c31b05%22%7D; x-web-ip=171.218.206.136, 47.121.110.187, 47.106.31.167, 100.121.99.209; .thumbcache_de0d870e3139ba2368b2e7ea8f11063c=hZVjd47Cv1sJHhGMmTvVn0f9DpivVn6IF/IytbVBOsjFSd0QEKKRar19TbnFCtp96Hi7/tlo0p6ovKPk4j5lOw%3D%3D; Hm_lpvt_8b5ea374777742fd73d0561bd1dfca6d=1747621872',
}

urls = []

with open("./url_value.txt", "r", encoding="utf-8") as file:
    urls = [url.strip() for url in file.readlines()]

items = []
lock = threading.Lock()  # 用于线程安全的列表操作

def scrape_page(url, index):
    proxy_pool = ProxyPool()
    max_retries = 3
    
    for retry in range(max_retries):
        proxy = proxy_pool.get_proxy()
        if not proxy:
            print(f"没有可用的代理，重试 {retry + 1}/{max_retries}")
            time.sleep(2)
            continue
            
        try:
            proxies = {
                "http": f"http://{proxy}",
                "https": f"http://{proxy}"
            }
            
            response = requests.get(
                url=url,
                cookies=cookies,
                headers=headers,
                proxies=proxies,
                timeout=10
            )
            response.raise_for_status()
            
            tree = etree.HTML(response.text)
            list_1 = []
            
            # 提取数据的代码保持不变
            Companyname_number = tree.xpath('//div[@class="copy-box initialShowCopyIcon"]/span/span[@class="text-num"]/text()')[0]
            Belonging_company = tree.xpath('//div[@class="cell flex-center value"]/span/span[@class="test"]/text()')[6]
            if bool(re.search(r'\d', Belonging_company)):
                 Belonging_company = tree.xpath('//div[@class="cell flex-center value"]/span/span[@class="test"]/text()')[5]
            else:
                pass
            try:
                uscc = tree.xpath('//div[@class="cell flex-center value fddb"]/span[@class="isPerson"]/a/text()')[0]
            except:
                uscc = tree.xpath('//div[@class="cell flex-center value fddb"]/span/span[@class="test"]/text()')[0]
            
            Number_employees = tree.xpath('//div[@class="cell flex-center value"]/span/span[@class="test"]/text()')[7]
            if bool(re.search(r'\d', Number_employees)):
                pass
            else:
                Number_employees = tree.xpath('//div[@class="cell flex-center value"]/span/span[@class="test"]/text()')[6]
            operating_revenue = tree.xpath('//div[@class="trade"]//div[@class="value"]/span/text()')[0]
            Establishment_time = tree.xpath('//div[@class="cell flex-center value f-1"]/span/span/text()')[0]
            registered_capital = tree.xpath('//div[@class="cell flex-center value"]/span/span[@class="test"]/text()')[1]
            Paid_capital = tree.xpath('//div[@class="cell flex-center value"]/span/span[@class="test"]/text()')[2]
            Enterprise_type = tree.xpath('//div[@class="cell flex-center value"]/span/span[@class="test"]/text()')[4]
            if '纳税人' in Enterprise_type:
                Enterprise_type = tree.xpath('//div[@class="cell flex-center value"]/span/span[@class="test"]/text()')[3]
            else:
                pass
            try:
                registered_address = tree.xpath('//div[@class="copy-box initialShowCopyIcon"]/span/div[@class="filter-source"]/text()')[0]
            except:
                registered_address = tree.xpath('//div[@class="copy-box initialShowCopyIcon"]/span/div[@class="filter-source no-value"]/text()')[0]
            brief_introduction = tree.xpath('//div[@class="company-detail-item"]/div[@class="text-box"]/div/p/text()')[0]
            business_scope = tree.xpath('//div[@class="flex"]/p[@class="value f-1"]/text()')[0]

            list_1.extend([Companyname_number, Belonging_company, uscc, Number_employees, 
                          operating_revenue, Establishment_time, registered_capital, 
                          Paid_capital, Enterprise_type, registered_address, brief_introduction,business_scope])
            
            with lock:
                items.append(list_1)
                print(f'已完成 {index}/{len(urls)}: {url}')
            
            return True
            
        except Exception as e:
            proxy_pool.delete_proxy(proxy)
            if retry < max_retries - 1:
                time.sleep(2)
            continue
    
    print(f'代理三次ip失败{index}/{len(urls)}: {url}')
    return False

# 多线程爬取
def main():
    start_time = time.time()
    
    # 使用线程池，max_workers控制并发线程数
    with ThreadPoolExecutor(max_workers=10) as executor:
        # 提交所有任务
        futures = {executor.submit(scrape_page, url, i): (url, i) 
                  for i, url in enumerate(urls, 1)}
        
        # 等待所有任务完成
        for future in as_completed(futures):
            url, i = futures[future]
            try:
                future.result()
            except Exception as e:
                print(f'任务异常: {url}, 错误: {e}')
    
    end_time = time.time()
    print(f"所有任务完成，耗时: {end_time - start_time:.2f}秒")
    # print("$"*30)
    
    # 保存数据到Excel
    df = pd.DataFrame(items, columns=['统一社会信用代码', '所属行业', '法定代表人', '员工人数', 
                                    '营业收入', '成立时间', '注册资本', '实缴资本', 
                                    '企业类型', '注册地址', '简介','经营范围'])
    df.to_excel('company_data.xlsx', index=False)
    print("数据已保存到 company_data.xlsx")

if __name__ == '__main__':
    main()